import pymysql
import requests
import json
import io
import sys

# 今日头条热搜通过js动态生成，下面地址为JS请求的数据接口，f12从network/xhr中获取
start_url = 'https://r.inews.qq.com/getWeiboRankingList?chlid=news_recommend_hot&appver=28_android_4.2.40&devid=&qn-rid=&qn-sig=f690e21095559203e3f55c42a04f8f15'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36 Edg/96.0.1054.43',
    'cookie': 'tab_info=%7B%22tab_ids%22%3A%5B%5D%2C%22tab_params%22%3A%7B%7D%7D'
}

# 改变标准输出的默认编码，防止spring boot运行时显示乱码
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')

# 保存到数据库
def saveDAO(db,number,topic,hot,href):
    cursor = db.cursor()
    query = """insert into tengxunxinwentop (ranking,topic,hot,url,daydate,mindate) values (%s,%s,%s,%s,curdate(),curtime())"""
    cursor.execute(query,(number,topic,hot,href))
    cursor.close()

# 进行requests请求
def get(url):
    response = requests.get(url, headers=headers)
    if response. status_code == 200:
        print("网页请求成功")
        parse(url)
    else:
        print("网页请求失败了")

# 将今日头条热搜的数据给爬下来
def parse(url):
    # 因为id自动递增，让自动增长从1开始，已经有了就跳过，让id连续
    db = pymysql.connect(host="localhost", user="root", password="root", database="hottop")
    cursor = db.cursor()
    str = 'ALTER TABLE tengxunxinwentop AUTO_INCREMENT = 1'
    cursor.execute(str)
    cursor.close()
    db.commit()

    # 由于是纯json数据，将其用文本形式保存
    response = requests.get(url, headers=headers)
    data = json.loads(response.text)
    # 获取json数据
    hotdata = data['idlist'][0]['newslist']
    hotdata.pop(0)
    try:
        for i in hotdata:
            number=i['hotEvent']['ranking']
            topic = i['hotEvent']['title']
            hot = i['hotEvent']['hotScore']
            href = "https://view.inews.qq.com/hotEvent/"+i['id']
#             print(number,topic, hot, href)
            saveDAO(db, number, topic, hot, href)
    except BaseException as ex:
        print(ex)
    else:
        db.commit()
        db.close()
    response.close()

#程序从此处开始运行
if __name__ == "__main__":
    get(start_url)
